import json
import os
import requests
from lxml import etree
import pymongo
import re
import threading

mongo_conn = pymongo.MongoClient()
counter = 0
headers = {
    "Accept": "*/*",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "Pragma": "no-cache",
    "Referer": "https://www.xinpianchang.com/discover/article-16-18-all-all-0-0-score-pp2",
    "Sec-Fetch-Dest": "empty",
    "Sec-Fetch-Mode": "cors",
    "Sec-Fetch-Site": "same-origin",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
    "sec-ch-ua": "\"Chromium\";v=\"130\", \"Google Chrome\";v=\"130\", \"Not?A_Brand\";v=\"99\"",
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": "\"Windows\""
}
cookies = {
    "Device_ID": "3i8fyc48lm216lhdg",
    "MEIQIA_TRACK_ID": "2nBBjy6vGLmtVbTq8KcQfAmHVUA",
    "MEIQIA_VISIT_ID": "2nBBjvM2aFubeWrmhoYqna0Gypn",
    "Authorization": "4EC46BA2681DB6A66681DB44C5681DBACB6681DB65EE9AFC6F64",
    "sensorsdata2015jssdkcross": "%7B%22distinct_id%22%3A%2215041561%22%2C%22first_id%22%3A%221926edb6166a-07d127846f2749-26001051-2073600-1926edb6167938%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_utm_source%22%3A%22xpcWeb%22%2C%22%24latest_utm_medium%22%3A%22navigatorCate%22%2C%22%24latest_utm_campaign%22%3A%22scjp%22%7D%2C%22%24device_id%22%3A%221926edb6166a-07d127846f2749-26001051-2073600-1926edb6167938%22%7D",
    "_ga": "GA1.2.1447999405.1729768816",
    "FECW": "31811a46fb46011dda3645e729a0b7a534e1d1ff8073d6011fb291767e13ef4671e5ebcbe8c3c2629e2db241a5132bc88793e7b51b9b5d09de16368583a5e971e1cabcee6b4391b0c90a1d5e14de17702e",
    "Hm_lvt_446567e1546b322b726d54ed9b5ad346": "1729682423,1729841524,1729905400,1730340805",
    "HMACCOUNT": "AFD53293F7A60F46",
    "PHPSESSID": "adcjsdegvrdk42hno3j3cd8377",
    "Hm_lpvt_446567e1546b322b726d54ed9b5ad346": "1730442052",
    "FECA": "6UeXZP+0qkCL/eXe9QGTuX2OQyxzdjeSkdSVtoJ67kAXmAhK19SgCA/Udkw93IVrUgDGUjAiiTRuPf8uDEHuD52A6M/w40WRDxZ86UfwFS/vhv65lIwA2Yk1oJ4aq6FpV8ljmuk2fnis6Xr+WpPcQXWFs0z82Qh3zMtdfkadZPmQBfkh+qnFVt/CrN4yKGhc2e"
}
url = "https://www.xinpianchang.com/api/xpc/v2/article/category"
params = {
    "FECU": "6UeXZP+0qkCL/eXe9QGTuX2OQyxzdjeSkdSVtoJ67kAXmAhK19SgCA/Udkw93IVrUgDGUjAiiTRuPf8uDEHuD52A6M/w40WRDxZ86UfwFS/vhv65lIwA2Yk1oJ4aq6FpV8ljmuk2fnis6Xr+WpPcQXREFvxfSbZQuKlJAebAPxjbEkoIvNu5QeowJqkrxklJde"
}
response = requests.get(url, headers=headers, cookies=cookies, params=params)

print(response.content.decode())
result = response.content.decode()
result = json.loads(result)
for category in result["data"]["list"]:
    # print(category,category["id"])
    os.makedirs(f"./新片场数据/{category['name']}", exist_ok=True)
    for second_category in category["children"]:

        os.makedirs(f"./新片场数据/{category['name']}/{second_category['name']}", exist_ok=True)
        page = 0
        while True:
            page += 1
            if page == 1:
                url = f"https://www.xinpianchang.com/_next/data/7iD9y4Ok2pZK53ds5V-ny/discover/article/{category['id']}-{second_category['id']}.json"
            else:
                url = f"https://www.xinpianchang.com/_next/data/7iD9y4Ok2pZK53ds5V-ny/discover/article/{category['id']}-{second_category['id']}-all-all-0-0-pick-pp{page}.json"
            headers = {
                "Accept": "*/*",
                "Accept-Language": "zh-CN,zh;q=0.9",
                "Cache-Control": "no-cache",
                "Connection": "keep-alive",
                "Pragma": "no-cache",
                "Referer": "https://www.xinpianchang.com/discover/article-1-355-all-all-0-0-pick",
                "Sec-Fetch-Dest": "empty",
                "Sec-Fetch-Mode": "cors",
                "Sec-Fetch-Site": "same-origin",
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
                "sec-ch-ua": "\"Chromium\";v=\"130\", \"Google Chrome\";v=\"130\", \"Not?A_Brand\";v=\"99\"",
                "sec-ch-ua-mobile": "?0",
                "sec-ch-ua-platform": "\"Windows\"",
                "x-nextjs-data": "1"
            }
            cookies = {
                "Device_ID": "3i8fyc48lm216lhdg",
                "MEIQIA_TRACK_ID": "2nBBjy6vGLmtVbTq8KcQfAmHVUA",
                "MEIQIA_VISIT_ID": "2nBBjvM2aFubeWrmhoYqna0Gypn",
                "sensorsdata2015jssdkcross": "%7B%22distinct_id%22%3A%2215041561%22%2C%22first_id%22%3A%221926edb6166a-07d127846f2749-26001051-2073600-1926edb6167938%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_utm_source%22%3A%22xpcWeb%22%2C%22%24latest_utm_medium%22%3A%22navigatorCate%22%2C%22%24latest_utm_campaign%22%3A%22scjp%22%7D%2C%22%24device_id%22%3A%221926edb6166a-07d127846f2749-26001051-2073600-1926edb6167938%22%7D",
                "_ga": "GA1.2.1447999405.1729768816",
                "FECW": "31811a46fb46011dda3645e729a0b7a534e1d1ff8073d6011fb291767e13ef4671e5ebcbe8c3c2629e2db241a5132bc88793e7b51b9b5d09de16368583a5e971e1cabcee6b4391b0c90a1d5e14de17702e",
                "Hm_lvt_446567e1546b322b726d54ed9b5ad346": "1729682423,1729841524,1729905400,1730340805",
                "HMACCOUNT": "AFD53293F7A60F46",
                "PHPSESSID": "adcjsdegvrdk42hno3j3cd8377",
                "Authorization": "C13D3888F150E9C7EF150E4A52F150E8160F150E574DDCACE586",
                "Hm_lpvt_446567e1546b322b726d54ed9b5ad346": "1730720606",
                "FECA": "6UeXZP+0qkCL/eXe9QGTuX2OQyxzdjeSkdSVtoJ67kAUd0BMJPC9t5/t6K3KIIMHacTOvOCdtdb9JA9nhX7VNo/VZ7zPhbNUIjiA2e9a9TDb6NO9fAMey9NKi5IZi1Os4QzNRbs5C7TOm6ZLxbBfNK1/kY+A1r85cf6EoWJwls4UTkOuorqFBg6urC33GcpqAe"
            }

            params = {
                "FECU": "6UeXZP+0qkCL/eXe9QGTuX2OQyxzdjeSkdSVtoJ67kAUd0BMJPC9t5/t6K3KIIMHacTOvOCdtdb9JA9nhX7VNo/VZ7zPhbNUIjiA2e9a9TDb6NO9fAMey9NKi5IZi1Os4QzNRbs5C7TOm6ZLxbBfNK5IDjeWa5vPZomGkh8rYff3jxoHQaFytTSsS3iDP+huie"
            }
            detail_response = requests.get(url, headers=headers, cookies=cookies, params=params)
            detail_result = detail_response.json()
            every_page_detail_list = detail_result['pageProps']['discoverArticleData']['list']
            for detail in every_page_detail_list:
                detail_list = []
                counter += 1
                try:
                    detail_author = f"{detail['author']['role']}+{detail['author']['userId']}"
                except:
                    detail_author = "无作者"
                detail_list.append(detail_author)
                print(detail_list)
                try:
                    detail_title = detail['title']
                except:
                    detail_title = "无标题"
                detail_list.append(detail_title)
                print(detail_list)
                try:
                    detail_url = detail['web_url']
                except:
                    detail_url = "无链接"
                detail_list.append(detail_url)
                count_collect = detail['count']['count_collect']
                count_comment = detail['count']['count_comment']
                count_like = detail['count']['count_like']
                count_share = detail['count']['count_share']
                count_view = detail['count']['count_view']
                publish_time = detail['publish_time']
                print(detail_list)
                detail_list = ",".join(detail_list)
                print(counter, category["id"], second_category["id"], second_category["name"])
                mongo_conn.xinpianchang.informations.insert_one({
                    "总数": counter,
                    "category": category["name"],
                    "second_category": second_category["name"],
                    "视频标题": detail_title,
                    "视频连接": detail_url,
                    "作者信息": detail_author,
                    "收藏数量": count_collect,
                    "评论数量": count_comment,
                    "点赞数量": count_like,
                    "分享数量": count_share,
                    "观看次数": count_view,
                    "发布时间": publish_time,
                })
            if detail_result['pageProps']['discoverArticleData']['next_page_url'] == "":
                break
